/*
Abstract:
  Analyze by words functionality implementation

Last changed:
  $Id: analyze_words.cpp 82 2009-12-08 20:53:44Z vitamin.caig $

Author:
  (C) Vitamin/CAIG/2001

  This file is the part of Textator project
*/

#include "analyzer.h"

#include <symbol_check.h>

#include <cassert>

#include <text.h>

namespace
{
  const String EMPTY;

  struct QuasiToken
  {
    QuasiToken(std::size_t pos) : Position(pos), RefCount(1), Type(Token::TEXT)
    {
    }
    std::size_t Position;
    std::size_t RefCount;
    Token::TokenType Type;
  };
  typedef std::map<String, QuasiToken> RefTokenMap;
  typedef std::vector<RefTokenMap::const_iterator> RefTokenList;
  typedef std::map<String, RefTokenList> RefTokens;

  bool IsFalse(String::value_type)
  {
    return false;
  }

  bool IsOther(String::value_type sym)
  {
    return !Symbol::IsAlpha(sym) && !Symbol::IsDigit(sym) && !Symbol::IsSpace(sym) && !Symbol::IsEscaped(sym);
  }

  class WordTokenizer
  {
  public:
    WordTokenizer()
    {
    }

    void Analyze(const StringMap& source, RefTokens& target)
    {
      for (StringMap::const_iterator it = source.begin(), lim = source.end(); it != lim; ++it)
      {
        ParseString(it->second, target[it->first]);
      }
    }

    void Process(const RefTokens& src, const Analyzer::ByWordsParameters& params, EntriesMap& result, Analyzer::Statistic::Tokenizing& statistic)
    {
      //process locals
      for (RefTokenMap::iterator it = Map.begin(), lim = Map.end(); it != lim; ++it)
      {
        if (it->second.RefCount >= params.MinWordsReferences && //enough references
            it->first.size() >= params.MinWordSize)
        {
          it->second.Type = Token::VARIABLE;
          //here it's text, but variable for further usage of iterator
          std::pair<EntriesMap::iterator, bool> tkIns(result.insert(
            EntriesMap::value_type(MakeName(it), Entry(Entry::LOCAL, Token::TEXT, it->first))));
          assert(tkIns.second || !"Invalid map");
          ++statistic.TokensCount;
          statistic.TokensSymbols += it->first.size();
          statistic.TokensReferences += it->second.RefCount;
          statistic.TokenizedSymbols += it->second.RefCount * it->first.size();
          statistic.MaxTokenSize = std::max(statistic.MaxTokenSize, it->first.size());
        }
      }
      //process globals
      for (RefTokens::const_iterator it = src.begin(), lim = src.end(); it != lim; ++it)
      {
        std::pair<EntriesMap::iterator, bool> tkIns(result.insert(
          EntriesMap::value_type(it->first, Entry(Entry::GLOBAL))));
        assert(tkIns.second || !"Invalid map");
        TokensList& tklist(tkIns.first->second.Value);
        if (it->second.empty())//process empty
        {
          tklist.push_back(Token(Token::TEXT, EMPTY));
          continue;
        }
        for (RefTokenList::const_iterator rit = it->second.begin(), rlim = it->second.end(); rit != rlim; ++rit)
        {
          if (Token::TEXT == (*rit)->second.Type)
          {
            if (!tklist.empty() &&
               tklist.back().Type == (*rit)->second.Type)
            {
              tklist.back().Value += (*rit)->first;//optimize
            }
            else
            {
              tklist.push_back(Token((*rit)->second.Type, (*rit)->first));
            }
          }
          else if (Token::VARIABLE == (*rit)->second.Type)
          {
            tklist.push_back(Token((*rit)->second.Type, MakeName(*rit)));
          }
          else
          {
            assert(!"Invalid type");
          }
        }
      }
    }
  private:
    String MakeName(RefTokenMap::const_iterator it) const
    {
      return Formatter<String>::Process(TEXT_FORMAT_TOKEN_NAME, it->second.Position);
    }
    void ParseString(const String& input, RefTokenList& output)
    {
      output.clear();
      output.reserve(input.size());//if any
      typedef bool (*CheckFunc)(String::value_type);
      CheckFunc checker(&IsFalse);
      String::const_iterator tknBeg(input.end());
      String::const_iterator it = input.begin(), lim = input.end();
      while (it != lim)
      {
        if (checker(*it))//same cathegory
        {
          ++it;
          continue;
        }
        if (tknBeg != input.end())
        {
          output.push_back(GetRefToken(String(tknBeg, it)));
        }
        tknBeg = it;
        if (((checker = &Symbol::IsAlpha, true) && checker(*it)) ||
            ((checker = &Symbol::IsDigit, true) && checker(*it)) ||
            ((checker = &Symbol::IsSpace, true) && checker(*it)) ||
            ((checker = &Symbol::IsEscaped, true) && checker(*it)) ||
            (checker = &IsOther, true)
           )
        {
          continue;
        }
        assert(!"Never reach it");
      }
      if (tknBeg != lim)
      {
        output.push_back(GetRefToken(String(tknBeg, lim)));
      }
    }

  private:
    RefTokenMap::const_iterator GetRefToken(const String& val)
    {
      std::pair<RefTokenMap::iterator, bool> tknIns(Map.insert(RefTokenMap::value_type(val, QuasiToken(Map.size()))));
      if (!tknIns.second)
      {
        ++tknIns.first->second.RefCount;
      }
      return tknIns.first;
    }
  private:
    RefTokenMap Map;
  };
}

namespace Analyzer
{
  //categories: spaces,printable,other
  void ByWords(const StringMap& source, const ByWordsParameters& params, EntriesMap& result, Statistic::Tokenizing& statistic)
  {
    RefTokens tmp;
    WordTokenizer tkzer;
    tkzer.Analyze(source, tmp);
    statistic = Statistic::Tokenizing();
    tkzer.Process(tmp, params, result, statistic);
  }
}
